{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "39329df3-1f99-4b11-9405-5969d52368a7",
   "metadata": {},
   "source": [
    "# Decision Tree & Random Search Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7f61a90e-a119-4bd0-af21-38604c5b4eec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scikit-learn: 1.0\n",
      "mlxtend     : 0.19.0\n",
      "xgboost     : 1.5.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -p scikit-learn,mlxtend,xgboost"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f0489c2-dd9c-4e71-a78c-e01201762b37",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "271b17ff-5ea4-4161-8b7f-20ba8131d666",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train/Valid/Test sizes: 398 80 171\n"
     ]
    }
   ],
   "source": [
    "from sklearn import model_selection\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import datasets\n",
    "\n",
    "\n",
    "data = datasets.load_breast_cancer()\n",
    "X, y = data.data, data.target\n",
    "\n",
    "X_train, X_test, y_train, y_test = \\\n",
    "    train_test_split(X, y, test_size=0.3, random_state=1, stratify=y)\n",
    "\n",
    "X_train_sub, X_valid, y_train_sub, y_valid = \\\n",
    "    train_test_split(X_train, y_train, test_size=0.2, random_state=1, stratify=y_train)\n",
    "\n",
    "print('Train/Valid/Test sizes:', y_train.shape[0], y_valid.shape[0], y_test.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c922b01-86f0-4e83-9e36-446f99f6fe1b",
   "metadata": {},
   "source": [
    "## Random Search"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0affc454-9f07-48e6-bcee-e6253d968247",
   "metadata": {},
   "source": [
    "## Randomized Search"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72e56f33-ec33-46dd-afa2-a1b3c8b3da0b",
   "metadata": {},
   "source": [
    "Example showing how to tune hyperparameters of a nested ensemble using randomized search \n",
    "- More info: \n",
    "  - https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.RandomizedSearchCV.html\n",
    "  - https://scikit-learn.org/stable/modules/grid_search.html#randomized-parameter-search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "96f0b4c1-803a-436f-93d5-31baab55faa5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 50 candidates, totalling 500 fits\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.48677424684556264, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.48677424684556264, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.48677424684556264, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.48677424684556264, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.48677424684556264, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.48677424684556264, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.48677424684556264, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.48677424684556264, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.48677424684556264, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.48677424684556264, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.20872664927422052, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.20872664927422052, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.20872664927422052, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.20872664927422052, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.20872664927422052, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.20872664927422052, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.20872664927422052, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.20872664927422052, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.20872664927422052, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.20872664927422052, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32080673889182354, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32080673889182354, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32080673889182354, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32080673889182354, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32080673889182354, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32080673889182354, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32080673889182354, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32080673889182354, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32080673889182354, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32080673889182354, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.483153718164036, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.483153718164036, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.483153718164036, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.483153718164036, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.483153718164036, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.483153718164036, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.483153718164036, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.483153718164036, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.483153718164036, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.483153718164036, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3579130122070837, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3579130122070837, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3579130122070837, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3579130122070837, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3579130122070837, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3579130122070837, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3579130122070837, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3579130122070837, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3579130122070837, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3579130122070837, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.11479122017688431, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.11479122017688431, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.11479122017688431, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.11479122017688431, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.11479122017688431, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.11479122017688431, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.11479122017688431, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.11479122017688431, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.11479122017688431, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.11479122017688431, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.26805168641259414, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.26805168641259414, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.26805168641259414, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.26805168641259414, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.26805168641259414, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.26805168641259414, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.26805168641259414, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.26805168641259414, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.26805168641259414, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.26805168641259414, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.07605915553713449, min_samples_split=7; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.07605915553713449, min_samples_split=7; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.07605915553713449, min_samples_split=7; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.07605915553713449, min_samples_split=7; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.07605915553713449, min_samples_split=7; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.07605915553713449, min_samples_split=7; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.07605915553713449, min_samples_split=7; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.07605915553713449, min_samples_split=7; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.07605915553713449, min_samples_split=7; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.07605915553713449, min_samples_split=7; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.14085672422492518, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.14085672422492518, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.14085672422492518, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.14085672422492518, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.14085672422492518, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.14085672422492518, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.14085672422492518, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.14085672422492518, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.14085672422492518, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.14085672422492518, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.03144804976857746, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.03144804976857746, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.03144804976857746, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.03144804976857746, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.03144804976857746, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.03144804976857746, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.03144804976857746, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.03144804976857746, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.03144804976857746, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.03144804976857746, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.002214914174985483, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.002214914174985483, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.002214914174985483, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.002214914174985483, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.002214914174985483, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.002214914174985483, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.002214914174985483, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.002214914174985483, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.002214914174985483, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.002214914174985483, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.05265542076628499, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.05265542076628499, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.05265542076628499, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.05265542076628499, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.05265542076628499, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.05265542076628499, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.05265542076628499, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.05265542076628499, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.05265542076628499, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.05265542076628499, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.428743009065693, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.428743009065693, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.428743009065693, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.428743009065693, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.428743009065693, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.428743009065693, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.428743009065693, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.428743009065693, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.428743009065693, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.428743009065693, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.26053448154587283, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.26053448154587283, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.26053448154587283, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.26053448154587283, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.26053448154587283, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.26053448154587283, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.26053448154587283, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.26053448154587283, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.26053448154587283, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.26053448154587283, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08919183207321685, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08919183207321685, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08919183207321685, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08919183207321685, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08919183207321685, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08919183207321685, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08919183207321685, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08919183207321685, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08919183207321685, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08919183207321685, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3126271664395207, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3126271664395207, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3126271664395207, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3126271664395207, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3126271664395207, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3126271664395207, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3126271664395207, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3126271664395207, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3126271664395207, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3126271664395207, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.12148526734715603, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.12148526734715603, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.12148526734715603, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.12148526734715603, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.12148526734715603, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.12148526734715603, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.12148526734715603, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.12148526734715603, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.12148526734715603, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.12148526734715603, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.15103916189854827, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.15103916189854827, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.15103916189854827, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.15103916189854827, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.15103916189854827, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.15103916189854827, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.15103916189854827, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.15103916189854827, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.15103916189854827, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.15103916189854827, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3928342958100834, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3928342958100834, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3928342958100834, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3928342958100834, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3928342958100834, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3928342958100834, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3928342958100834, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3928342958100834, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3928342958100834, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3928342958100834, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.30439613257875664, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.30439613257875664, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.30439613257875664, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.30439613257875664, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.30439613257875664, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.30439613257875664, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.30439613257875664, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.30439613257875664, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.30439613257875664, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.30439613257875664, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.42063069810061166, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.42063069810061166, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.42063069810061166, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.42063069810061166, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.42063069810061166, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.42063069810061166, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.42063069810061166, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.42063069810061166, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.42063069810061166, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.42063069810061166, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4477044325686145, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4477044325686145, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4477044325686145, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4477044325686145, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4477044325686145, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4477044325686145, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4477044325686145, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4477044325686145, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4477044325686145, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4477044325686145, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.08688016919081087, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.08688016919081087, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.08688016919081087, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.08688016919081087, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.08688016919081087, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.08688016919081087, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.08688016919081087, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.08688016919081087, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.08688016919081087, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.08688016919081087, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08973430161007023, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08973430161007023, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08973430161007023, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08973430161007023, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08973430161007023, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08973430161007023, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08973430161007023, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08973430161007023, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08973430161007023, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.08973430161007023, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3890526432550406, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3890526432550406, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3890526432550406, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3890526432550406, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3890526432550406, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3890526432550406, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3890526432550406, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3890526432550406, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3890526432550406, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.3890526432550406, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.49486877070519464, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.49486877070519464, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.49486877070519464, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.49486877070519464, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.49486877070519464, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.49486877070519464, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.49486877070519464, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.49486877070519464, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.49486877070519464, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.49486877070519464, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.06015001530116293, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.06015001530116293, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.06015001530116293, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.06015001530116293, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.06015001530116293, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.06015001530116293, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.06015001530116293, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.06015001530116293, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.06015001530116293, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.06015001530116293, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.11828725979727978, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.11828725979727978, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.11828725979727978, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.11828725979727978, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.11828725979727978, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.11828725979727978, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.11828725979727978, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.11828725979727978, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.11828725979727978, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.11828725979727978, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2940908463186484, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2940908463186484, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2940908463186484, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2940908463186484, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2940908463186484, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2940908463186484, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2940908463186484, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2940908463186484, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2940908463186484, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2940908463186484, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.031010311234937138, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.031010311234937138, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.031010311234937138, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.031010311234937138, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.031010311234937138, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.031010311234937138, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.031010311234937138, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.031010311234937138, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.031010311234937138, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.031010311234937138, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4678191776301691, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4678191776301691, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4678191776301691, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4678191776301691, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4678191776301691, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4678191776301691, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4678191776301691, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4678191776301691, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4678191776301691, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4678191776301691, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.2733280941824238, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.2733280941824238, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.2733280941824238, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.2733280941824238, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.2733280941824238, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.2733280941824238, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.2733280941824238, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.2733280941824238, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.2733280941824238, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.2733280941824238, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.04454543124587118, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.04454543124587118, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.04454543124587118, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.04454543124587118, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.04454543124587118, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.04454543124587118, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.04454543124587118, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.04454543124587118, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.04454543124587118, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.04454543124587118, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.10342121063745702, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.10342121063745702, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.10342121063745702, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.10342121063745702, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.10342121063745702, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.10342121063745702, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.10342121063745702, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.10342121063745702, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.10342121063745702, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.10342121063745702, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3756122947480083, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3756122947480083, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3756122947480083, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3756122947480083, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3756122947480083, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3756122947480083, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3756122947480083, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3756122947480083, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3756122947480083, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.3756122947480083, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.13840961382711364, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.13840961382711364, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.13840961382711364, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.13840961382711364, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.13840961382711364, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.13840961382711364, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.13840961382711364, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.13840961382711364, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.13840961382711364, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.13840961382711364, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.19473778965146843, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.19473778965146843, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.19473778965146843, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.19473778965146843, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.19473778965146843, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.19473778965146843, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.19473778965146843, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.19473778965146843, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.19473778965146843, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.19473778965146843, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.40693316688871295, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.40693316688871295, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.40693316688871295, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.40693316688871295, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.40693316688871295, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.40693316688871295, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.40693316688871295, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.40693316688871295, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.40693316688871295, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.40693316688871295, min_samples_split=6; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.16519972375263164, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.16519972375263164, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.16519972375263164, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.16519972375263164, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.16519972375263164, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.16519972375263164, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.16519972375263164, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.16519972375263164, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.16519972375263164, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.16519972375263164, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.22721268153900903, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.22721268153900903, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.22721268153900903, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.22721268153900903, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.22721268153900903, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.22721268153900903, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.22721268153900903, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.22721268153900903, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.22721268153900903, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.22721268153900903, min_samples_split=8; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4239874499994308, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4239874499994308, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4239874499994308, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4239874499994308, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4239874499994308, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4239874499994308, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4239874499994308, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4239874499994308, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4239874499994308, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.4239874499994308, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.3061341138709028, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.3061341138709028, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.3061341138709028, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.3061341138709028, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.3061341138709028, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.3061341138709028, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.3061341138709028, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.3061341138709028, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.3061341138709028, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.3061341138709028, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.4739298930195651, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.4739298930195651, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.4739298930195651, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.4739298930195651, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.4739298930195651, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.4739298930195651, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.4739298930195651, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.4739298930195651, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.4739298930195651, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.4739298930195651, min_samples_split=3; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32170662045533993, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32170662045533993, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32170662045533993, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32170662045533993, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32170662045533993, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32170662045533993, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32170662045533993, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32170662045533993, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32170662045533993, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.32170662045533993, min_samples_split=4; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.2971101882024814, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.2971101882024814, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.2971101882024814, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.2971101882024814, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.2971101882024814, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.2971101882024814, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.2971101882024814, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.2971101882024814, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.2971101882024814, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=6, min_impurity_decrease=0.2971101882024814, min_samples_split=11; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.1259221682905926, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.1259221682905926, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.1259221682905926, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.1259221682905926, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.1259221682905926, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.1259221682905926, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.1259221682905926, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.1259221682905926, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.1259221682905926, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.1259221682905926, min_samples_split=9; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.17241120345074828, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.17241120345074828, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.17241120345074828, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.17241120345074828, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.17241120345074828, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.17241120345074828, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.17241120345074828, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.17241120345074828, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.17241120345074828, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=16, min_impurity_decrease=0.17241120345074828, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2604155509707317, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2604155509707317, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2604155509707317, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2604155509707317, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2604155509707317, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2604155509707317, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2604155509707317, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2604155509707317, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2604155509707317, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2604155509707317, min_samples_split=10; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.04180282719037426, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.04180282719037426, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.04180282719037426, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.04180282719037426, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.04180282719037426, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.04180282719037426, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.04180282719037426, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.04180282719037426, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.04180282719037426, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.04180282719037426, min_samples_split=2; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2729462781412829, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2729462781412829, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2729462781412829, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2729462781412829, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2729462781412829, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2729462781412829, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2729462781412829, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2729462781412829, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2729462781412829, min_samples_split=5; total time=   0.0s\n",
      "[CV] END max_depth=None, min_impurity_decrease=0.2729462781412829, min_samples_split=5; total time=   0.0s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9298717948717948"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import scipy.stats\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "\n",
    "clf = DecisionTreeClassifier(random_state=123)\n",
    "\n",
    "params =  {\n",
    "    'min_samples_split': scipy.stats.randint(2, 12),\n",
    "    'min_impurity_decrease': scipy.stats.uniform(0.0, 0.5),\n",
    "    'max_depth': [6, 16, None]\n",
    "}\n",
    "\n",
    "\n",
    "search = RandomizedSearchCV(\n",
    "    estimator=clf,\n",
    "    param_distributions=params,\n",
    "    n_iter=50,\n",
    "    cv=10,\n",
    "    verbose=2,\n",
    "    n_jobs=1)\n",
    "\n",
    "\n",
    "search.fit(X_train, y_train)\n",
    "\n",
    "search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2c26399d-ebfc-4b06-86d9-36e49711e908",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 6,\n",
       " 'min_impurity_decrease': 0.002214914174985483,\n",
       " 'min_samples_split': 2}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "763e816b-6437-45a9-812f-8b429472d75e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 1.00\n",
      "Test Accuracy: 0.94\n"
     ]
    }
   ],
   "source": [
    "print(f\"Training Accuracy: {search.best_estimator_.score(X_train, y_train):0.2f}\")\n",
    "print(f\"Test Accuracy: {search.best_estimator_.score(X_test, y_test):0.2f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
